{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-ab1dd681009b>:3: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From C:\\Anaconda\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From C:\\Anaconda\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting mnist/input_data\\train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Anaconda\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting mnist/input_data\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Anaconda\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting mnist/input_data\\t10k-images-idx3-ubyte.gz\n",
      "Extracting mnist/input_data\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From C:\\Anaconda\\envs\\python3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "# Import data\n",
    "data_dir = 'mnist/input_data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 构建神经网络输入层与一层隐层\n",
    "x1 = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "n1=256     #隐层神经元数，经尝试设为270\n",
    "\n",
    "#按截断式正态分布初始化权重与偏置\n",
    "W1 = tf.Variable(tf.truncated_normal([784, n1],stddev=0.1))\n",
    "b1 = tf.Variable(tf.truncated_normal([n1] ,stddev=0.1)) #参数stddev为随机初始化时，截断正态分布的方差，均值默认为0\n",
    "\n",
    "logits1 = tf.matmul(x1, W1) + b1\n",
    "\n",
    "y1 = tf.nn.relu(logits1)  #激活函数取relu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-4-c6d985df15ad>:10: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# 构建输出层\n",
    "W2 = tf.Variable(tf.truncated_normal([n1, 10],stddev=0.1))\n",
    "b2 = tf.Variable(tf.truncated_normal([10],stddev=0.1))  #参数stddev为随机初始化时，截断正态分布的方差，经调整设为0.1，均值默认为0\n",
    "\n",
    "logits2 = tf.matmul(y1, W2) + b2\n",
    "\n",
    "y2 = tf.nn.relu(logits2)\n",
    "\n",
    "#交叉熵\n",
    "cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_ , logits=logits2))\n",
    "\n",
    "#正则项\n",
    "regularizer=tf.contrib.layers.l2_regularizer(scale=0.0001, scope=None)  #正则参数经尝试调整，设为0.0001\n",
    "L2=tf.contrib.layers.apply_regularization(regularizer, weights_list=[W1,W2])\n",
    "\n",
    "#总的损失函数\n",
    "loss=tf.add(cross_entropy , L2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.3486 \t step:  0\n",
      "accuracy:  0.9488 \t step:  300\n",
      "accuracy:  0.9648 \t step:  600\n",
      "accuracy:  0.97 \t step:  900\n",
      "accuracy:  0.9722 \t step:  1200\n",
      "accuracy:  0.9754 \t step:  1500\n",
      "accuracy:  0.9763 \t step:  1800\n",
      "accuracy:  0.978 \t step:  2100\n",
      "accuracy:  0.9791 \t step:  2400\n",
      "accuracy:  0.98 \t step:  2700\n",
      "learning rate is 0.5,get accuracy:  0.9799\n"
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y2, 1), tf.argmax(y_ , 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "train_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss) #学习率初步调整为0.5\n",
    "\n",
    "# 以0.4为初始学习率，开始第一次网络训练\n",
    "for i in range(3000):   \n",
    "    batch_xs, batch_ys = mnist.train.next_batch(256)     #batch数与隐层神经元数经尝试取为相同\n",
    "    sess.run(train_step, feed_dict={x1: batch_xs, y_: batch_ys})\n",
    "    if i%300==0:                              #每300次循环输出一次当前准确率\n",
    "        acc = sess.run(accuracy, feed_dict={x1: mnist.test.images,y_: mnist.test.labels})\n",
    "        print (\"accuracy: \",acc,\"\\t\",\"step: \",i)\n",
    "        \n",
    "        \n",
    "# Test trained model\n",
    "print(\"learning rate is 0.5,get accuracy: \",sess.run(accuracy, feed_dict={x1: mnist.test.images, y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy:  0.9823 \t step:  3000\n",
      "accuracy:  0.9824 \t step:  3300\n",
      "accuracy:  0.9823 \t step:  3600\n",
      "accuracy:  0.9821 \t step:  3900\n",
      "accuracy:  0.9823 \t step:  4200\n",
      "accuracy:  0.9824 \t step:  4500\n",
      "accuracy:  0.9823 \t step:  4800\n",
      "accuracy:  0.9828 \t step:  5100\n",
      "accuracy:  0.9823 \t step:  5400\n",
      "accuracy:  0.9821 \t step:  5700\n",
      "learning rate is 0.15,continue the training,get final accuracy:  0.9822\n"
     ]
    }
   ],
   "source": [
    "# 将学习率调低至0.15，继续网络训练3000个step\n",
    "train_step = tf.train.GradientDescentOptimizer(0.15).minimize(loss) #学习率调整为0.3\n",
    "\n",
    "for i in range(3000,6000):   \n",
    "    batch_xs, batch_ys = mnist.train.next_batch(256)     #batch数与隐层神经元数经尝试取为相同\n",
    "    sess.run(train_step, feed_dict={x1: batch_xs, y_: batch_ys})\n",
    "    if i%300==0:                              #每300次循环输出一次当前准确率\n",
    "        acc = sess.run(accuracy, feed_dict={x1: mnist.test.images,y_: mnist.test.labels})\n",
    "        print (\"accuracy: \",acc,\"\\t\",\"step: \",i)\n",
    "\n",
    "# Test trained model\n",
    "print(\"learning rate is 0.15,continue the training,get final accuracy: \",\n",
    "      sess.run(accuracy, feed_dict={x1: mnist.test.images, y_: mnist.test.labels}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
